const axios = require("axios");
const cheerio = require("cheerio");
const iconv = require("iconv-lite");
const fs = require("fs");

function startSpider(url, config, ouput = "data.json", encoding = "utf8") {
  // 拿到url对应的html
  axios
    .get(url, {
      responseType: "arraybuffer",
    })

    .then((res) => {
      const html = iconv.decode(res.data, encoding);
      // console.log("data=", html);

      // 按照用户配置 从html中爬取数据
      spiderIt(html, config, ouput);
    })

    // 处理错误
    .catch((err) => console.log("err=", err));
}

const spiderIt = (html, config, ouput) => {
  // 准备好数据数组
  let dataArr = [];

  // node环境下无法使用DOM-API,BOM-API
  // cheerio自带一个虚拟的DOM-API
  const $ = cheerio.load(html);

  for (let key in config) {
    // 获取到配置：selector=定位承载value的DOM元素 cb/dom=>value为从DOM元素中提取value的方法
    const { selector, cb } = config[key];

    // 找出所有承载value的元素
    const doms = $(selector);

    doms.each((index, dom) => {
      // console.log("item",item);
      // 加载首个key之前 为dataArr中填充数量相等的空对象
      !dataArr[index] && (dataArr[index] = {});

      // 伪代码：dataArr[0].src = img.getAttribute("src")
      dataArr[index][key] = cb($(dom));
    });
  }

  const jsonData = JSON.stringify(dataArr);
  console.log("jsonData", jsonData);

  fs.writeFile(ouput, jsonData, () => console.log("输出成功"));
};

/* 
想要在爬取的结果形成如下数据格式：
[
  {
    "src":"https://game.gtimg.cn/images/daoju/app/lol/medium/pkg-32449.jpg?_t=1652244674",
    "strong":"EDG 不破不立 大礼包",
    "price":"571.20"
  },
  ...
]

列表item中key的名称:{
  selector:定位到承载数据的DOM元素,
  cb:从DOM元素中抠出数据的方法 (dom)=>value
}
*/
startSpider(
  "https://daoju.qq.com/lol/",
  {
    src: {
      selector: ".listbox>dl>dt>a>img",
      cb: (img) => img.attr("src"),
    },
    strong: {
      selector: ".listbox>dl>dd>a>strong",
      cb: (strong) => strong.text(),
    },
    price: {
      selector: ".listbox>dl>dd>.bxlist-t1>span",
      cb: (span) => span.eq(0).text().trim(),
    },
  },
  "inventory.json",
  "gbk"
);
